package com.chance.cc.crawler.development.scripts.douyin.api;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.zip.GZIPOutputStream;

import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_Domain_Result_Json;
import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

/**
 * @author bx
 * @date 2020/12/6 0006 12:52
 */
public class DYDevicesRegisterScript extends CrawlerCommonScript {

    private static Logger logger = LoggerFactory.getLogger(DYDevicesRegisterScript.class);
    public static final String site = "api-device";
    public static final String domain = "api-dy";

    private Downloader httpDownload;
    private static Object httpDownloadObj = new Object();
    private static HttpConfig commonConfig = HttpConfig.me("common");


    public static final String startUrlRegular = "https://start/dy/devices/register";

    private static final String hookApi = "http://192.168.1.212:8385";
    public static final String saveDeviceApi = "http://192.168.1.215:9599";
    private static Proxy proxy = new Proxy();
    static {
        //代理配置
//        proxy.setHost("http-cla.abuyun.com");
//        proxy.setPort(9030);
//        proxy.setUsername("H3487178Q0I1HVPC");
//        proxy.setPassword("ACE81171D81169CA");
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    private static HttpConfig dyConfig = HttpConfig.me(domain);
    static {
        dyConfig.setProxy(proxy);
        dyConfig.setUseProxy(true);
        dyConfig.setDisableCookie(true);
    }
    public static final String localDeviceFileFormat = "/data/chance_crawler_test/data/douyin/devices/devices_%s.txt";
//    public static final String localDeviceFileFormat = "C:\\Users\\Administrator\\Desktop\\jar\\org\\devices_%s.txt";
    private static LinkedBlockingQueue<String> deviceTypeQueue = new LinkedBlockingQueue<>();
    static {
        deviceTypeQueue.add("device_account");
        deviceTypeQueue.add("device_search");
        deviceTypeQueue.add("device_comment");
        deviceTypeQueue.add("device_comment");
        deviceTypeQueue.add("device_comment");
        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
//        deviceTypeQueue.add("device_comment");
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        String devicesInfo = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(Tag_Field_Domain_Result_Json);
        if (StringUtils.isNotBlank(devicesInfo)){
            Json devicesInfoJson = new Json(devicesInfo);
            String build_serial = devicesInfoJson.jsonPath($_type + ".serialno").get();
            String openudid = devicesInfoJson.jsonPath($_type + ".android").get();
            String mac = devicesInfoJson.jsonPath($_type + ".mac").get();
            device_register(build_serial,openudid,mac);
        }
        return new ArrayList<>();
    }
    private void device_register(String build_serial, String openudid, String address){

        String os_version = "7.1.2";
        int os_api = 25;
        String device_model = "SM-G9550";
        String device_brand = "samsung";
        String device_manufacturer = "samsung";
        String cpu_abi = "armeabi-v7a";
        String resolution = "720x1280";
        String mc = address.toUpperCase();
        String mcc_mnc = "46007";

        device_register(openudid,os_version,os_api,device_model,device_brand,
                device_manufacturer,cpu_abi,resolution,build_serial,mc,mcc_mnc);


    }

    private void device_register(String openudid,String os_version,
                                        int os_api,String device_model,String device_brand,String device_manufacturer,
                                        String cpu_abi,String resolution,String build_serial,String mc,String mcc_mnc){

        String cdid = UUID.randomUUID().toString();
        String oaid = RandomStringUtils.random(16,"abcdef1234567890");


        String clientudid = UUID.randomUUID().toString();
        String req_id1 = UUID.randomUUID().toString();
        String req_id2 = UUID.randomUUID().toString();
        long time = System.currentTimeMillis()-1000*60*1L;
        long apk_first_install_time = System.currentTimeMillis()-1000*60*5L;
        long _gen_time = System.currentTimeMillis();
        String channel = "huawei_"+ RandomUtils.nextInt(20,90);
        String urlFormat = "https://log3-misc.amemv.com/service/2/device_register/?" +
                "ac=wifi&channel=%s&aid=1128&app_name=aweme&version_code=140100&version_name=14.1.0&device_platform=android" +
                "&ssmix=a&device_type=%s&device_brand=%s&language=zh&os_api=%s&os_version=%s" +
                "&openudid=%s&manifest_version_code=140101&resolution=%s&dpi=480&update_version_code=14109900" +
                "&appTheme=dark&mcc_mnc=%s&cpu_support64=true&host_abi=%s&app_type=normal&cdid=%s&oaid=%s&tt_data=a";

        String url = String.format(urlFormat,channel,device_model,device_brand,os_api,os_version,openudid,resolution,mcc_mnc,cpu_abi, cdid, oaid);

        Map<String,Object> requestBody = new HashMap<>();
        Map<String,Object> header = new HashMap<>();
        Map<String,Object> oaidMap = new HashMap<>();
        requestBody.put("magic_tag","ss_app_log");
        requestBody.put("_gen_time",_gen_time);
        requestBody.put("header",header);

        oaidMap.put("req_id",req_id1);
        oaidMap.put("hw_id_version_code","null");
        oaidMap.put("take_ms","43");
        oaidMap.put("is_track_limited","false");
        oaidMap.put("query_times","1");
        oaidMap.put("id",oaid);
        oaidMap.put("time",time);

        String channelValue = RandomStringUtils.random(8,"abcdefghijklmnopqqrsduvwxyz1234567890_");
//        String channelValue = "huawei_"+ RandomUtils.nextInt(1,10);
        header.put("display_name","抖音");
        header.put("update_version_code","14109900");
        header.put("manifest_version_code","140101");
        header.put("app_version_minor","140101");
        header.put("aid","1128");
        header.put("channel",channelValue);
        header.put("appkey","57bfa27c67e58e7d920028d3");
        header.put("package","com.ss.android.ugc.aweme");
        header.put("app_version","14.1.0");
        header.put("version_code","140100");
        header.put("sdk_version","2.14.0-rc.3-backoff");
        header.put("sdk_target_version",29);
        header.put("git_hash","490e7989");
        header.put("os","Android");
        header.put("os_version",os_version);
        header.put("os_api",os_api);
        header.put("device_model",device_model);
        header.put("device_brand",device_brand);
        header.put("device_manufacturer",device_manufacturer);
        header.put("cpu_abi",cpu_abi);
        header.put("build_serial",build_serial);
        header.put("release_build","804c624_20201222_24b2d6fa-4403-11eb-bfa5-02420a000038");
        header.put("density_dpi","280");
        header.put("display_density","mdpi");
        header.put("resolution",resolution);
        header.put("language","zh");
        header.put("mc",mc);
        header.put("timezone",8);
        header.put("access","wifi");
        header.put("not_request_sender",0);
        header.put("carrier","中国电信");
        header.put("mcc_mnc",mcc_mnc);
//        header.put("rom","MIUI-V11.0.2.0.OAGCNXM");
//        header.put("rom_version","MIUI-V11.0.2.0.OAGCNXM");
        header.put("cdid",cdid);
        header.put("sig_hash","aea615ab910015038f73c47e45d21466");
        header.put("openudid",openudid);
        header.put("clientudid",clientudid);
        header.put("serial_number",build_serial);
        header.put("sim_serial_number",new ArrayList<>());
        header.put("region","CN");
        header.put("tz_name","Asia/Shanghai");
        header.put("tz_offset",28800);
        header.put("sim_region","cn");
        header.put("req_id",req_id2);
        header.put("oaid",oaidMap);
        header.put("apk_first_install_time",apk_first_install_time);
        header.put("is_system_app",0);
        header.put("sdk_flavor","china");
        String param = JSON.toJSONString(requestBody);
        logger.info("devices register param {}",param);
        StringBuffer sb = new StringBuffer();
        for(byte b : toBytes(param)){
            sb.append(b).append(",");
        }
        String str_data = sb.substring(0,sb.length()-1);
        String registerResult = dyRegister(str_data);
        registerResult = registerResult.substring(1,registerResult.length()-1);
        String result = deviceRegist(url, registerResult);
        Map map = JSON.parseObject(result, Map.class);
        String device_id = map.get("device_id").toString();
        String iid = map.get("install_id").toString();
        if ("0".equals(device_id) || "0".equals(iid)){
            logger.error("devices register androidID {} fail", openudid);
        } else {
//            String rawText = downloadTest( cdid, device_id, device_model, iid, openudid, String.valueOf(os_api), os_version, resolution);
            Map<String,String> deviceMap = new HashMap<>();
            deviceMap.put("openudid",openudid);
            deviceMap.put("os_version",os_version);
            deviceMap.put("os_api",String.valueOf(os_api));
            deviceMap.put("device_model",device_model);
            deviceMap.put("device_brand",device_brand);
            deviceMap.put("device_manufacturer",device_manufacturer);
            deviceMap.put("cpu_abi",cpu_abi);
            deviceMap.put("resolution",resolution);
            deviceMap.put("build_serial",build_serial);
            deviceMap.put("mc",mc);
            deviceMap.put("mcc_mnc",mcc_mnc);
            deviceMap.put("device_id",device_id);
            deviceMap.put("iid",iid);
            deviceMap.put("cdid",cdid);
            deviceMap.put("oaid",oaid);
            deviceMap.put("clientudid",clientudid);
            deviceMap.put("channel",channelValue);

            //保存设备信息
            String category = deviceTypeQueue.poll();
            deviceTypeQueue.add(category);
            deviceMap.put("category",category);
            saveDevice(JSON.toJSONString(deviceMap));
            logger.info("save device {} success!",JSON.toJSONString(deviceMap));

//            if (StringUtils.isNotBlank(rawText)){
//                JSONObject jsonObject = JSONObject.parseObject(rawText);
//                JSONArray awemeList = jsonObject.getJSONArray("aweme_list");
//                if (null != awemeList && awemeList.size() > 0){
//                    //保存设备信息
//                    String category = deviceTypeQueue.poll();
//                    deviceTypeQueue.add(category);
//                    deviceMap.put("category",category);
//                    saveDevice(JSON.toJSONString(deviceMap));
//                    logger.info("save device {} success!",JSON.toJSONString(deviceMap));
//                }else {
//                    logger.error("devices register androidID {} fail,download test result {}, devices info {}",
//                            openudid,rawText,JSON.toJSONString(deviceMap));
//                }
//
//            }else {
//                logger.error("devices register androidID {} fail,download test result {}, devices info {}",
//                        openudid,rawText,JSON.toJSONString(deviceMap));
//            }
        }
    }


    private String downloadTest(String cdid,String device_id,
                                       String device_type,String iid,String openudid,String os_api,String os_version, String resolution){

//        HttpConfig httpConfig = HttpConfig.me("accountTest1");
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(true);
        dyConfig.setHttpSite("account_test");
        String channel = "huawei_"+ RandomUtils.nextInt(20,90);

        String common_param_v2 = "aid=1128&app_name=aweme&app_type=normal&cdid=%s&channel="+channel+"&device_brand=samsung" +
                "&device_id=%s&device_platform=android&device_type=%s&dpi=480&iid=%s&language=zh&manifest_version_code=140101" +
                "&openudid=%s&os_api=%s&os_version=%s&resolution=%s&ssmix=a&update_version_code=14109900" +
                "&version_code=140100&version_name=14.1.0";

        HttpRequest httpRequest = new HttpRequest();
        long rticket = System.currentTimeMillis();
        long ts = rticket / 1000;

        String accountUrlParamFormat = "source=0&user_avatar_shrink=96_96&video_cover_shrink=248_330" +
                "&publish_video_strategy_type=2&max_cursor=0" +
                "&sec_user_id=MS4wLjABAAAAWFV4r4bm27prbxuX5xE7JljwCD5vh4SN3V_VxZd-_tQ&count=20" +
                "&is_order_flow=0&page_from=2&longitude=121.5888983509045&latitude=31.25251444047919" +
                "&location_permission=true&_rticket="+rticket+"&cpu_support64=true&host_abi=armeabi-v7a" +
                "&appTheme=dark&ac=wifi&ts="+ts+"&";

        String url = "https://api3-core-c-lf.amemv.com/aweme/v1/aweme/post/?" + accountUrlParamFormat;
        httpRequest.setUrl(url);

        String common_params = String.format(common_param_v2,cdid,device_id,device_type,iid,openudid,os_api,os_version,resolution);
        logger.info("x common params v2 {}",common_params);

        httpRequest.addHeader("User-Agent", "com.ss.android.ugc.aweme/140101 (Linux; U; Android 7.1.2; zh_CN; SM-G9550;" +
                " Build/OPR1.170623.032; Cronet/TTNetVersion:"+RandomStringUtils.random(8,"0123456789abcdf")+" 2020-12-16 QuicVersion:"+RandomStringUtils.random(8,"0123456789abcdf")+" 2020-10-14)");
        httpRequest.addHeader("x-common-params-v2", common_params);
        Map<String,String> dataMap = new HashMap<>();
        Map<String, String> signatureMap = dySignature( url,dataMap);

        httpRequest.getHeaders().putAll(signatureMap);
        HttpPage download = null;
        String rawText = null;
        try {
            download = httpDownload.download(httpRequest,dyConfig);
            if (!download.isDownloadSuccess()){
                throw new Exception("download error!");
            }
            rawText = download.getRawText();
        } catch (Exception e) {
            try {
                while (download == null || !download.isDownloadSuccess()){
                    Thread.sleep(3000);
                    download = httpDownload.download(httpRequest, dyConfig);
                    rawText = download.getRawText();
                }
            } catch (Exception ex) {
                logger.error(ex.getMessage());
            }
        }

        return rawText;
    }

    private void saveDevice(String deviceJson) {
        //写入本地文件-防止数据丢失。
        try {
            String date = DateFormatUtils.format(System.currentTimeMillis(),"yyyyMMdd");
            FileOutputStream fileOutputStream = new FileOutputStream(String.format(localDeviceFileFormat, date), true);
            IOUtils.writeLines(Arrays.asList(deviceJson),"\r\n",fileOutputStream,"utf-8");
        } catch (IOException e) {
            logger.error("write devices to local file fail,error {}",e.getMessage());
        }

        //写入mysql
        String saveUrl = saveDeviceApi + "/crawler/oauth/api/v1/addOrUpdateUserOAuthInfo?isDeleteAll=false";
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(saveUrl);
        httpRequest.setMethod(HttpConstant.Method.POST);
        Map<String,String> userOauthInfoMap = new HashMap<>();
        userOauthInfoMap.put("domain","douyin");
        userOauthInfoMap.put("oauthInfo", deviceJson);
        httpRequest.setRequestBody(HttpRequestBody.json(JSON.toJSONString(Arrays.asList(userOauthInfoMap)),"utf-8"));
        HttpPage download = httpDownload.download(httpRequest,commonConfig);
        logger.info("save device result :"+ download.getRawText());
    }



    private byte[] toBytes(String param){
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        GZIPOutputStream gzip;
        try {
            gzip = new GZIPOutputStream(out);
            gzip.write(param.getBytes());
            gzip.close();
        } catch (IOException e) {
        }
        byte[] bytes = out.toByteArray();
        return bytes;
    }

    private String dyRegister(String str_data){
        String registerUrl = hookApi + "/frida/dy/register";
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(registerUrl);
        Map<String,Object> dyDataMap = new HashMap<>();
        dyDataMap.put("str_data",str_data);
        httpRequest.setRequestBody(HttpRequestBody.form(dyDataMap,"utf-8"));
        httpRequest.setMethod("POST");
        String rawText = null;
        HttpPage download = null;
        try {
            download = httpDownload.download(httpRequest, commonConfig);
            rawText = download.getRawText();
            if(rawText.contains("DOCTYPE HTML PUBLIC")){
                throw new Exception("DOCTYPE HTML PUBLIC");
            }
        } catch (Exception e) {
            try {
                while (download == null || !download.isDownloadSuccess()){
                    Thread.sleep(60000);
                    download = httpDownload.download(httpRequest, commonConfig);
                    rawText = download.getRawText();
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
        return rawText;
    }

    private String deviceRegist(String url, String byteStr){
//        HttpConfig httpConfig = HttpConfig.me("deviceRegister1");
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(true);
        dyConfig.setHttpSite("devices");
        String[] split = byteStr.split(",");
        byte[] bytes = new byte[split.length];
        for (int i = 0; i < split.length; i++) {
            bytes[i] = (byte)Integer.parseInt(split[i]);
        }
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setMethod("POST");
        httpRequest.setUrl(url);
        httpRequest.setRequestBody(HttpRequestBody.custom(bytes,"application/octet-stream;tt-data=a","utf-8"));

        Map<String,String> dataMap = Maps.newHashMap();
        try {
            String stub = md5(bytes);
            Map<String, String> signatureMap = dySignature(url, dataMap);
            httpRequest.addHeader("User-Agent", "com.ss.android.ugc.aweme/140101 (Linux; U; Android 7.1.2; zh_CN; SM-G9550;" +
                    " Build/OPR1.170623.032; Cronet/TTNetVersion:"+RandomStringUtils.random(8,"0123456789abcdf")+" 2020-12-16 QuicVersion:"+RandomStringUtils.random(8,"0123456789abcdf")+" 2020-10-14)");
            httpRequest.addHeader("Content-Type", "application/octet-stream;tt-data=a");
            httpRequest.getHeaders().putAll(signatureMap);
            httpRequest.addHeader("X-SS-STUB", stub);

        } catch (NoSuchAlgorithmException e) {
            logger.error(e.getMessage());
        }
        String rawText = null;
        HttpPage download = null;
        try {
            download = httpDownload.download(httpRequest,dyConfig);
            if (!download.isDownloadSuccess()){
                throw new Exception("download error!");
            }
            rawText = download.getRawText();
        } catch (Exception e) {
            try {
                while (download == null || !download.isDownloadSuccess() || StringUtils.isBlank(rawText)){
                    Thread.sleep(8000);
                    download = httpDownload.download(httpRequest, dyConfig);
                    rawText = download.getRawText();
                }
            } catch (Exception ex) {
                logger.error(ex.getMessage());
            }
        }
        return rawText;
    }

    private Map<String,String> dySignature(String dy_url,Map<String,String> dataMap){
        String signatureUrl = hookApi + "/frida/dy/signature";

        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(signatureUrl);
        Map<String,Object> dyDataMap = new HashMap<>();
        dyDataMap.put("dy_url",dy_url);
        dyDataMap.put("map_data_json",JSON.toJSONString(dataMap));
        httpRequest.setRequestBody(HttpRequestBody.form(dyDataMap,"utf-8"));
        httpRequest.setMethod("POST");
        String rawText = null;
        HttpPage download = null;
        try {
            download = httpDownload.download(httpRequest, commonConfig);
            rawText = download.getRawText();
            if(rawText.contains("DOCTYPE HTML PUBLIC")){
                throw new Exception("DOCTYPE HTML PUBLIC");
            }
        } catch (Exception e) {
            try {
                while (download == null || !download.isDownloadSuccess() || StringUtils.isBlank(rawText)){
                    Thread.sleep(60000);
                    download = httpDownload.download(httpRequest, commonConfig);
                    rawText = download.getRawText();
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
        return JSON.parseObject(rawText,Map.class);
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        return null;
    }

    @Override
    public String domain() {
        return "api-dy";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(startUrlRegular); //start url--只是为了进入脚本没有实际意义

    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {

        String recordSite = crawlerRecord.tagsCreator().bizTags().site();
        if (site.equals(recordSite)){
            return true;
        }
        return false;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        downloadSleep(); //请求间隔8s
        return super.prepareRequest(requestRecord, supportSourceRecords);
    }

    private void downloadSleep(){
        Random rand = new Random();
        long sleepTime = (long)((rand.nextFloat()+7)*1000L);
        logger.info("dy devices register download sleep time {}",sleepTime);
        try {
            Thread.sleep(sleepTime);
        } catch (InterruptedException e) {
            logger.error(e.getMessage());
        }
    }

    @Override
    public void beforeDownload(CrawlerRecordContext context) {
        if (httpDownload == null){
            synchronized (httpDownloadObj) {
                if (httpDownload == null){
                    httpDownload = context.getPageDownloader();
                }
            }
        }
        super.beforeDownload(context);
    }

    private void addCrawlerRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){

        int count = 1;
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        if (crawlerBusinessTags.hasKVTag("download_retry_count")){
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag("download_retry_count").getVal();
            if (count >= 500){
                logger.error("dy api account download he number of retries exceeds the limit" +
                        ",request url {},download detail {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        count++;
        crawlerBusinessTags.addCustomKV("download_retry_count",count);

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpRequest(crawlerRecord.getHttpRequest())
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .notFilterRecord()
                .needWashed(true)
                .build();
        crawlerRequestRecords.add(crawlerRequestRecord);

    }
}
